{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "84823e7f",
   "metadata": {},
   "source": [
    "# Data Exploration VisDrone"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94d45db9",
   "metadata": {},
   "source": [
    "## VisDrone Annotated Data Format\n",
    "\n",
    "`<bbox_left>, <bbox_top>, <bbox_width>, <bbox_height>, <score>, <object_category>, <truncation>, <occlusion>`\n",
    "\n",
    "classes:  ignore(0), pedestrian(1), people(2), bicycle(3), car(4), van(5), truck(6), tricycle(7), awning-tricycle(8), bus(9), motor(10), others(11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00868c3c",
   "metadata": {},
   "source": [
    "# uncomment below to use ipyml\n",
    "# %matplotlib widget "
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d0a836ed",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:29:09.754117Z",
     "start_time": "2023-03-03T10:29:09.749518Z"
    }
   },
   "source": [
    "import os\n",
    "import glob\n",
    "import math\n",
    "from typing import List, Tuple, Union, Iterable, Dict\n",
    "from collections import Counter, defaultdict\n",
    "\n",
    "import cv2\n",
    "import imagesize\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib import ticker\n",
    "\n",
    "# import custom packages\n",
    "import sys\n",
    "sys.path.append(\"..\") \n",
    "from utils.common import get_annot_img_paths, load_visdrone_annots_as_np\n",
    "\n",
    "ANNOT_EXT = {\".txt\"}\n",
    "IMG_EXT = {\".jpg\", \".png\"}\n",
    "CLASS_ID2LABEL = {0:\"ignore\", 1:\"pedestrian\", 2:\"people\", 3:\"bicycle\", 4:\"car\", 5:\"van\", \n",
    "                  6:\"truck\", 7:\"tricycle\", 8:\"awning-tricycle\", 9:\"bus\", 10:\"motor\", 11:\"others\"}"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f3a9f95d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T09:13:26.722650Z",
     "start_time": "2023-03-03T09:13:26.719302Z"
    }
   },
   "source": [
    "# choose data paths\n",
    "root_data_dir = \"../../data/visdrone_data/OriginalVisDrone/\"\n",
    "train_annots_dir = os.path.join(root_data_dir, \"VisDrone2019-DET-train/annotations\")\n",
    "train_images_dir = os.path.join(root_data_dir, \"VisDrone2019-DET-train/images\")\n",
    "val_annots_dir = os.path.join(root_data_dir, \"VisDrone2019-DET-val/annotations\")\n",
    "val_images_dir = os.path.join(root_data_dir, \"VisDrone2019-DET-val/images\")\n",
    "test_annots_dir = os.path.join(root_data_dir, \"VisDrone2019-DET-test-dev/annotations\")\n",
    "test_images_dir = os.path.join(root_data_dir, \"VisDrone2019-DET-test-dev/images\")\n",
    "\n",
    "# classes to consider\n",
    "CLASS_IDS_2_CONSIDER = {1, 2, 4, 5, 6, 9}"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7908bd2f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T09:13:28.317920Z",
     "start_time": "2023-03-03T09:13:28.280181Z"
    }
   },
   "source": [
    "train_annot_paths, train_image_paths = get_annot_img_paths(train_annots_dir, train_images_dir, ANNOT_EXT, IMG_EXT)\n",
    "val_annot_paths, val_image_paths = get_annot_img_paths(val_annots_dir, val_images_dir, ANNOT_EXT, IMG_EXT)\n",
    "test_annot_paths, test_image_paths = get_annot_img_paths(test_annots_dir, test_images_dir, ANNOT_EXT, IMG_EXT)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b5112667",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T09:13:30.980625Z",
     "start_time": "2023-03-03T09:13:30.906873Z"
    },
    "code_folding": []
   },
   "source": [
    "imgs_loaded = 9\n",
    "drawn_imgs = []\n",
    "\n",
    "for i, (ant, img) in enumerate(zip(test_annot_paths, test_image_paths)):\n",
    "    if i == imgs_loaded:\n",
    "        break\n",
    "    image = cv2.imread(img)\n",
    "    annots = load_visdrone_annots_as_np(ant)\n",
    "    for annot in annots:\n",
    "        score, class_id = annot[4], annot[5]\n",
    "        if class_id not in CLASS_IDS_2_CONSIDER:  # car, van ,truck, bus\n",
    "            continue\n",
    "        color = (0, 0, 255) if score == 0 else (0, 255, 0)\n",
    "        x1, y1, x2, y2 = annot[:4]\n",
    "        cv2.rectangle(image, (x1, y1), (x2, y2), color, 1)\n",
    "    drawn_imgs.append(image[:,:,::-1])"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "5cba9f8b",
   "metadata": {},
   "source": [
    "## Investigate bbox annotated images "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7df5dea8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T09:15:07.478942Z",
     "start_time": "2023-03-03T09:15:07.342998Z"
    }
   },
   "source": [
    "plt.figure(figsize=(8, 8))\n",
    "plt.imshow(drawn_imgs[2], interpolation='nearest')\n",
    "plt.show()"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "576de49b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T09:15:09.356334Z",
     "start_time": "2023-03-03T09:15:08.640188Z"
    }
   },
   "source": [
    "n_row, n_col = 3, 3\n",
    "_, axs = plt.subplots(n_row, n_col, figsize=(12, 12))\n",
    "axs = axs.flatten()\n",
    "\n",
    "for img, ax in zip(drawn_imgs[:imgs_loaded], axs):\n",
    "    ax.imshow(img)\n",
    "    ax.tick_params(axis='x', labelsize=6)\n",
    "    ax.tick_params(axis='y', labelsize=6)\n",
    "plt.show()"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "3d862921",
   "metadata": {},
   "source": [
    "## Qualitative Observations from inspecting annotated images\n",
    "\n",
    "1. Annotated Bbox overlap multiple objects / lots of occulsion in bboxes\n",
    "2. Bbox scales very from extremely small to decently size (requires statistical histogram binning check)\n",
    "3. Some images have zero instances of relevant classes\n",
    "4. Lighting conditions vary (Night, Day, evening) & some images are blurry\n",
    "5. Viewing angle ranges from elevated sideways (looking down at a slanted angle) to straight down (satellite view but higher resolution)\n",
    "6. Objects (Cars, trucks) are rotated at all angles (0-360), esp in straight-down views so bbox width/heights have high variation\n",
    "7. Highly imbalanced in favor of cars & some images have dense number of objects (cars)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80ee2492",
   "metadata": {},
   "source": [
    "## Quantitative Analysis\n",
    "\n",
    "-  Number of images, bboxes, classes\n",
    "-  Image Max min avg height\n",
    "-  Image Max min avg width\n",
    "-  Histogram binning of image heights & widths across entire dataset\n",
    "-  Distribution of number of objects across images\n",
    "-  Class wise distribution of objects across entire dataset\n",
    "-  Class wise bounding box area as a perc of total area distribution with histogram binning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "179a9723",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:02:41.972427Z",
     "start_time": "2023-03-03T10:02:41.955804Z"
    },
    "code_folding": [
     105,
     119
    ]
   },
   "source": [
    "def generate_data_stats_and_dict(annot_paths: List[str], image_paths: List[str]) -> Dict:\n",
    "    assert len(annot_paths) == len(image_paths)\n",
    "\n",
    "    annots = [load_visdrone_annots_as_np(p) for p in annot_paths]\n",
    "    image_shapes = [imagesize.get(p) for p in image_paths]\n",
    "    \n",
    "    image_heights = [h for _, h in image_shapes]\n",
    "    image_widths = [w for w, _ in image_shapes]\n",
    "    object_ids_per_img = []\n",
    "    bbox_area_percs = []\n",
    "    bbox_classes = []\n",
    "    bbox_widths = []\n",
    "    bbox_heights = []\n",
    "    used_for_eval = 0  # when score = 1\n",
    "    \n",
    "    filter_class_arr = np.array(list(CLASS_IDS_2_CONSIDER))\n",
    "    for i, annot in enumerate(annots):\n",
    "        # only get classes from CLASS_IDS_2_CONSIDER\n",
    "        annot = annot[(annot[:,5,None] == filter_class_arr[None,:]).any(axis=1)]\n",
    "        \n",
    "        object_ids_per_img.extend(annot.shape[0] * [i])\n",
    "        used_for_eval += (sum(annot[:, 4]))\n",
    "        w, h = image_shapes[i]\n",
    "        image_area = h * w\n",
    "        bbox = annot[:, :4]\n",
    "        bwidths, bheights = bbox[:, 2] - bbox[:, 0], bbox[:, 3] - bbox[:, 1]\n",
    "        bbox_classes.extend(annot[:, 5])\n",
    "        bbox_widths.extend(bwidths.tolist())\n",
    "        bbox_heights.extend(bheights.tolist())\n",
    "        bbox_area = bwidths * bheights\n",
    "        bbox_area_perc = 100 * bbox_area/image_area\n",
    "        bbox_area_percs.extend(bbox_area_perc.tolist())\n",
    "    \n",
    "    bbox_areas = [h*w for h, w in zip(bbox_heights, bbox_widths)]\n",
    "    idict = {}\n",
    "    idict[\"image_heights\"] = image_heights\n",
    "    idict[\"image_widths\"] = image_widths\n",
    "    idict[\"object_ids_per_img\"] = object_ids_per_img\n",
    "    idict[\"bbox_classes\"] = bbox_classes\n",
    "    idict[\"bbox_widths\"] = bbox_widths\n",
    "    idict[\"bbox_heights\"] = bbox_heights\n",
    "    idict[\"bbox_area_pixels\"] = bbox_areas    \n",
    "    idict[\"bbox_area_percs\"] = bbox_area_percs\n",
    "    \n",
    "    print(f\"Number of classes: {len(np.unique(bbox_classes)), np.unique(bbox_classes)}\")\n",
    "    print(\"Used for eval fraction (score=1)\", used_for_eval/len(bbox_area_percs))\n",
    "    print(f\"Number of images: {len(image_paths)}\")\n",
    "    print(f\" Image height: min {min(image_heights)}, max {max(image_heights)}, avg {sum(image_heights)/len(image_heights):.2f}\")\n",
    "    print(f\" Image width: min {min(image_widths)}, max {max(image_widths)}, avg {sum(image_widths)/len(image_widths):.2f}\")\n",
    "    print(f\"Number of bboxes: {len(object_ids_per_img)}\")\n",
    "    print(f\" BBox height: min {min(bbox_heights)}, max {max(bbox_heights)}, avg {sum(bbox_heights)/len(bbox_heights):.2f}\")\n",
    "    print(f\" BBox width: min {min(bbox_widths)}, max {max(bbox_widths)}, avg {sum(bbox_widths)/len(bbox_widths):.2f}\")\n",
    "    print(f\" BBox height percentiles (5%, 25%, 50%, 75%, 95%) {np.percentile(bbox_heights, (5, 25, 50, 75, 95))}\")\n",
    "    print(f\" BBox width percentiles (5%, 25%, 50%, 75%, 95%) {np.percentile(bbox_widths, (5, 25, 50, 75, 95))}\")\n",
    "    print(f\" BBox Area percentiles (33%, 35%, 50%, 66%, 85%) {np.percentile(bbox_areas, (33, 35, 50, 66, 85))}\")\n",
    "\n",
    "    return idict\n",
    "\n",
    "def remove_outliers_idict_inplace(\n",
    "    idict: Dict,\n",
    "    lower_dim_inc_cutoff: float = None,\n",
    "    lower_area_inc_cutoff: float = None,\n",
    ") -> None:\n",
    "    \"\"\"\n",
    "    Cutoff is only done by dim or area or by both\n",
    "    lower_dim_inc_cutoff is in pixels\n",
    "    lower_area_inc_cutoff is entered as percentage of total image area i.e. 0.1, 1, etc.\n",
    "    \"\"\"\n",
    "    hkey, wkey, clskey  = \"bbox_heights\", \"bbox_widths\", \"bbox_classes\"\n",
    "    blen = len(idict[hkey])\n",
    "    print(\"Before Cutoff\")\n",
    "    print(f\"\\tTotal number of boxes: {blen}\")\n",
    "    class_count_before = sorted([[CLASS_ID2LABEL[k], v] for k,v in Counter(idict[clskey]).items()])\n",
    "    print(\"\\tClass Counts:\", class_count_before)\n",
    "\n",
    "    idxs_dim = np.full(blen, True)\n",
    "    idxs_area = np.full(blen, True)\n",
    "    if lower_dim_inc_cutoff is not None:\n",
    "        hw = np.asarray([np.asarray(idict[hkey]), np.asarray(idict[wkey])])\n",
    "        idxs_dim = np.all(hw >= lower_dim_inc_cutoff, axis=0)\n",
    "    if lower_area_inc_cutoff is not None:\n",
    "        idxs_area = np.asarray(idict[\"bbox_area_percs\"]) >= lower_area_inc_cutoff\n",
    "    idxs = np.logical_and(idxs_dim, idxs_area)\n",
    "\n",
    "    # do the cutoffs\n",
    "    idict[\"object_ids_per_img\"] = np.asarray(idict[\"object_ids_per_img\"])[idxs]\n",
    "    idict[clskey] = np.asarray(idict[clskey])[idxs]\n",
    "    idict[wkey] = np.asarray(idict[wkey])[idxs]\n",
    "    idict[hkey] = np.asarray(idict[hkey])[idxs]\n",
    "    idict[\"bbox_area_pixels\"] = np.asarray(idict[\"bbox_area_pixels\"])[idxs]\n",
    "    idict[\"bbox_area_percs\"] = np.asarray(idict[\"bbox_area_percs\"])[idxs]\n",
    "    \n",
    "    print(\"After Cutoff\")\n",
    "    print(f\"\\tTotal number of boxes: {len(idict[hkey])}\")\n",
    "    class_count_after = sorted([[CLASS_ID2LABEL[k], v] for k,v in Counter(idict[clskey]).items()])\n",
    "    print(\"\\tClass Counts:\", class_count_after)\n",
    "    \n",
    "    reduction = [[b[0], f\"{a[1]/b[1] * 100:.2f}%\"] for b, a in zip(class_count_before, class_count_after)]\n",
    "    print(\"Remaining class counts after reduction\", reduction)\n",
    "    if lower_dim_inc_cutoff is not None:\n",
    "        print(f\"With a lower width/height dim cutoff value of {lower_dim_inc_cutoff} pixels:\")\n",
    "    if lower_area_inc_cutoff is not None:\n",
    "        print(f\"With a lower bbox area cutoff value of {lower_area_inc_cutoff} %:\")\n",
    "    print(f\"\\t{100*sum(idxs)/len(idxs):.2f}% of values kept\")\n",
    "    print(f\"\\tBbox dimensions [min, max]: Width [{min(idict[wkey]), max(idict[wkey])}], Height {min(idict[hkey]), max(idict[hkey])}\")\n",
    "\n",
    "def disp_histograms(\n",
    "    title: str,\n",
    "    xlabel: str,\n",
    "    ylabel: str,\n",
    "    data_labels: List[str],\n",
    "    data_list: List[Iterable],\n",
    "    use_logs: bool = False,\n",
    "    nbins: int = 50,\n",
    "    fsize: Tuple[int, int] = (8, 8),\n",
    "    alpha: float = 0.5,\n",
    "    xtick_rotation: float = 90,\n",
    "    xtick_fontsize: int = 8,\n",
    "    edgecolor: Tuple[int, int, int] = (0, 0, 0),\n",
    "    DPI: int = 100\n",
    "):\n",
    "    assert len(data_labels) == len(data_list)\n",
    "    min_d, max_d = float('inf'), float('-inf')\n",
    "    for data in data_list:\n",
    "        min_d = min(min(data), min_d)\n",
    "        max_d = max(max(data), max_d)\n",
    "    # fixed num of bins\n",
    "    bins = np.linspace(math.ceil(min_d), math.floor(max_d), nbins)\n",
    "\n",
    "    plt.figure(figsize=fsize, dpi=DPI)\n",
    "    plt.xlim([min_d-1, max_d+1])\n",
    "    plt.xticks(np.arange(min_d, max_d+1, 4*(max_d-min_d)//nbins),\n",
    "               rotation=xtick_rotation, fontsize=xtick_fontsize)\n",
    "    plt.title(title)\n",
    "    plt.xlabel(xlabel)\n",
    "    plt.ylabel(ylabel + (\" (log10)\" if use_logs else \"\"))\n",
    "    for data, label in zip(data_list, data_labels):\n",
    "        data = np.log10(np.asarray(data)) if use_logs else data\n",
    "        plt.hist(data, bins, alpha=alpha, label=label, edgecolor=edgecolor)\n",
    "    if any(data_labels):\n",
    "        plt.legend(loc='upper right')\n",
    "    \n",
    "    plt.show()"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "46339073",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:25:38.798515Z",
     "start_time": "2023-03-03T10:25:37.856458Z"
    }
   },
   "source": [
    "print(\"\\nTrain data statistics:\")\n",
    "idict_train = generate_data_stats_and_dict(train_annot_paths, train_image_paths)\n",
    "\n",
    "print(\"\\nVal data statistics:\")\n",
    "idict_val = generate_data_stats_and_dict(val_annot_paths, val_image_paths)\n",
    "\n",
    "print(\"\\nTest data statistics:\")\n",
    "idict_test = generate_data_stats_and_dict(test_annot_paths, test_image_paths)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "848d0001",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:08:29.076475Z",
     "start_time": "2023-03-03T10:08:29.060298Z"
    }
   },
   "source": [
    "train_class_count_before_cutoff = Counter(idict_train[\"bbox_classes\"])"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "162cb0ff",
   "metadata": {},
   "source": [
    "## Distritbution of bbox areas with relation to classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3ddb0127",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:34:00.204772Z",
     "start_time": "2023-03-03T10:33:59.943556Z"
    }
   },
   "source": [
    "plt.figure(figsize=(10,8))\n",
    "plt.set_cmap('jet')\n",
    "p = np.arange(25, 100, 10)\n",
    "\n",
    "obj_cls_areas = defaultdict(list)\n",
    "for cls, area in zip(idict_train[\"bbox_classes\"], idict_train[\"bbox_area_percs\"]):\n",
    "    obj_cls_areas[cls].append(area)\n",
    "\n",
    "for k in sorted(obj_cls_areas.keys(), key=lambda k: len(obj_cls_areas.get(k))):\n",
    "    v = obj_cls_areas.get(k)\n",
    "    v.sort()\n",
    "    plt.semilogx(v, np.arange(len(v))/len(v)*100, label=CLASS_ID2LABEL.get(k))\n",
    "    \n",
    "plt.grid()\n",
    "plt.legend()\n",
    "formatter = ticker.FuncFormatter(lambda y, _: '{:g}'.format(y))\n",
    "plt.gca().xaxis.set_major_formatter(formatter)\n",
    "plt.gca().yaxis.set_major_formatter(formatter)\n",
    "plt.xlabel('percentage area\\nof total image')\n",
    "plt.ylabel('percentile')\n",
    "plt.show()"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "08822230",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:34:42.872298Z",
     "start_time": "2023-03-03T10:34:42.773168Z"
    }
   },
   "source": [
    "cutoff_perc = .05\n",
    "\n",
    "plt.figure(figsize=(6,5))\n",
    "class_names = list(map(lambda x: CLASS_ID2LABEL.get(x), obj_cls_areas.keys()))\n",
    "y_below = list(map(lambda x: len(list(filter(lambda y: y < cutoff_perc, x))), obj_cls_areas.values()))\n",
    "y_above = list(map(lambda x: len(list(filter(lambda y: y >= cutoff_perc, x))), obj_cls_areas.values()))\n",
    "plt.barh(class_names, y_above, label='selected')\n",
    "plt.barh(class_names, y_below, left=y_above, label='total')\n",
    "for i, (v1, v2) in enumerate(zip(y_above, y_below)):\n",
    "    plt.gca().text(v1 + 3, i + .25, str(round(v1/1e3, 1)), color='blue')\n",
    "    plt.gca().text(v1 + v2 + 3, i - .25, str(round((v1+v2)/1e3, 1)), color='red')\n",
    "plt.xlabel('count(x1000)')\n",
    "formatter = ticker.FuncFormatter(lambda y, _: f'{int(y/1e3)}')\n",
    "plt.gca().xaxis.set_major_formatter(formatter)\n",
    "plt.legend()\n",
    "plt.gca().set_axisbelow(True)\n",
    "plt.grid(axis='x')\n",
    "plt.show()\n",
    "\n",
    "selected_perc = [(above / (above + below)) for above, below in zip(y_above, y_below)]\n",
    "print([f\"{cname}: {p * 100:.2f}%\" for cname, p in zip(class_names, selected_perc)])\n",
    "print(f\"With a percentage cutoff of {cutoff_perc}%, {100 * sum(y_above) / (sum(y_above) + sum(y_below)):.2f}% of values kept\")\n"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "54f59d0c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:08:29.282741Z",
     "start_time": "2023-03-03T10:08:29.077277Z"
    }
   },
   "source": [
    "# filter bboxes that are very small/outliers but keep large bboxes\n",
    "lower_dim_cutoff = 20\n",
    "lower_area_cutoff = 0.05\n",
    "print(\"\\nFiltering train bboxes based on bbox dims\")\n",
    "remove_outliers_idict_inplace(idict_train, lower_dim_inc_cutoff=lower_dim_cutoff, lower_area_inc_cutoff=lower_area_cutoff)\n",
    "print(\"\\nFiltering val bboxes based on bbox dims\")\n",
    "remove_outliers_idict_inplace(idict_val, lower_dim_inc_cutoff=lower_dim_cutoff, lower_area_inc_cutoff=lower_area_cutoff)\n",
    "print(\"\\nFiltering test bboxes based on bbox dims\")\n",
    "remove_outliers_idict_inplace(idict_test, lower_dim_inc_cutoff=lower_dim_cutoff, lower_area_inc_cutoff=lower_area_cutoff)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "3b4b7f83",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:08:36.845924Z",
     "start_time": "2023-03-03T10:08:36.840698Z"
    }
   },
   "source": [
    "perc_3 = np.percentile(idict_train[\"bbox_area_pixels\"], (33, 50, 85))\n",
    "\n",
    "print(f' BBox Area percentiles (33%, 50%, 66%) {\"total area:\",perc_3, \"dim:\", [round(p**(1/2)) for p in perc_3]}')"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "df988490",
   "metadata": {},
   "source": [
    "### Statistical Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a8b753cb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:22:14.042721Z",
     "start_time": "2023-03-03T10:22:13.953721Z"
    }
   },
   "source": [
    "class_bbox_count_list_before_cutoff = sorted([[k, CLASS_ID2LABEL[k], v]\n",
    "                               for k, v in train_class_count_before_cutoff.items()])\n",
    "class_name_before_cutoff = [k for _, k, _ in class_bbox_count_list_before_cutoff]\n",
    "class_counts_before_cutoff = [v for _, _, v in class_bbox_count_list_before_cutoff]\n",
    "\n",
    "class_bbox_counts_after_cutoff = Counter(idict_train[\"bbox_classes\"])\n",
    "class_bbox_count_list_after_cutoff = sorted([[k, CLASS_ID2LABEL[k], v]\n",
    "                               for k, v in class_bbox_counts_after_cutoff.items()])\n",
    "class_name_after_cutoff = [k for _, k, _ in class_bbox_count_list_after_cutoff]\n",
    "class_counts_after_cutoff = [v for _, _, v in class_bbox_count_list_after_cutoff]\n",
    "\n",
    "retained_perc = sum(class_counts_after_cutoff) / sum(class_counts_before_cutoff)\n",
    "# fig = plt.figure(figsize=(8,6))\n",
    "fig, ax = plt.subplots(figsize=(8,6))\n",
    "barb = plt.bar(class_name_before_cutoff, class_counts_before_cutoff, color='red', width=0.2, align='edge')\n",
    "bara = plt.bar(class_name_after_cutoff, class_counts_after_cutoff, color='green', width=0.2, align='edge')\n",
    "\n",
    "reduction = [f\"{a/b * 100:.2f}%\" for i, (b, a) in enumerate(zip(class_counts_before_cutoff, class_counts_after_cutoff))]\n",
    "# Add percs above the two bar graphs\n",
    "for i, rect in enumerate(bara):\n",
    "    height = rect.get_height()\n",
    "    plt.text(rect.get_x() + rect.get_width() / 2.0, height, f'{reduction[i]}', ha='center', va='bottom', fontsize=\"medium\", fontweight =\"bold\")\n",
    "\n",
    "plt.xticks(rotation=30)\n",
    "plt.xlabel(\"Class\")\n",
    "plt.ylabel(\"No. of bounding boxes\")\n",
    "plt.title(f\"Train bounding box class dist. \\nwith boxes with dimension <{lower_dim_cutoff} pixels removed and bbox area <{lower_area_cutoff}% removed\")\n",
    "plt.text(0.605, 0.95, f\"{retained_perc*100:.2f}% of total values kept\", transform=ax.transAxes, fontweight=\"bold\", fontsize=\"large\", backgroundcolor=(.9,.9,.9))\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "30cfaa2c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:11:48.199737Z",
     "start_time": "2023-03-03T10:11:48.034293Z"
    }
   },
   "source": [
    "disp_histograms(title = \"Train bounding box width & height dist.\", \n",
    "                xlabel = \"Bbox size (pixels)\", \n",
    "                ylabel = \"Number of bboxes\",\n",
    "                data_labels = [\"Bbox heights\", \"Bbox widths\"],\n",
    "                data_list =[idict_train[\"bbox_heights\"], idict_train[\"bbox_widths\"]],\n",
    "                nbins=80)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "1e555b7b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:06:44.169998Z",
     "start_time": "2023-03-03T10:06:44.017554Z"
    }
   },
   "source": [
    "disp_histograms(title = \"Val bounding box width & height dist.\", \n",
    "                xlabel = \"Bbox size (pixels)\", \n",
    "                ylabel = \"Number of bboxes\",\n",
    "                data_labels = [\"Bbox heights\", \"Bbox widths\"],\n",
    "                data_list =[idict_val[\"bbox_heights\"], idict_val[\"bbox_widths\"]],\n",
    "                nbins=80)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "45aa37ad",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:06:45.971139Z",
     "start_time": "2023-03-03T10:06:45.729240Z"
    }
   },
   "source": [
    "disp_histograms(title = \"Test bounding box width & height dist.\", \n",
    "                xlabel = \"Bbox size (pixels)\", \n",
    "                ylabel = \"Number of bboxes\",\n",
    "                data_labels = [\"Bbox heights\", \"Bbox widths\"],\n",
    "                data_list =[idict_test[\"bbox_heights\"], idict_test[\"bbox_widths\"]],\n",
    "                nbins=80)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "cf61c367",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:06:47.188406Z",
     "start_time": "2023-03-03T10:06:46.941868Z"
    }
   },
   "source": [
    "disp_histograms(title =\"Objects per image dist.\", \n",
    "                xlabel = \"Number of objects\", \n",
    "                ylabel = \"Number of images\",\n",
    "                data_labels = [\"train\", \"val\", \"test\"],\n",
    "                data_list =[list(Counter(idict_train[\"object_ids_per_img\"]).values()), \n",
    "                            list(Counter(idict_val[\"object_ids_per_img\"]).values()), \n",
    "                            list(Counter(idict_test[\"object_ids_per_img\"]).values())],\n",
    "                nbins=100,\n",
    "                alpha=0.3,\n",
    "                edgecolor=None)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "c8f750a0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:06:48.832162Z",
     "start_time": "2023-03-03T10:06:48.700032Z"
    }
   },
   "source": [
    "disp_histograms(title =\"Train bbox area dist.\", \n",
    "                xlabel = \"Bbox Area % (As a perc of total image area)\", \n",
    "                ylabel = \"Number of bboxes\",\n",
    "                data_labels = [None],\n",
    "                data_list =[idict_train[\"bbox_area_percs\"]],\n",
    "                nbins=70,\n",
    "                use_logs=False,\n",
    "                edgecolor=None)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "1560f410",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:06:50.654657Z",
     "start_time": "2023-03-03T10:06:50.535612Z"
    }
   },
   "source": [
    "disp_histograms(title =\"Val bbox area dist.\", \n",
    "                xlabel = \"Bbox Area % (As a perc of total image area)\", \n",
    "                ylabel = \"Number of bboxes\",\n",
    "                data_labels = [None],\n",
    "                data_list =[idict_val[\"bbox_area_percs\"]],\n",
    "                nbins=70,\n",
    "                use_logs=False,\n",
    "                edgecolor=None)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "1631c0fb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-03T10:06:52.954914Z",
     "start_time": "2023-03-03T10:06:52.838356Z"
    }
   },
   "source": [
    "disp_histograms(title =\"Test bbox area dist.\", \n",
    "                xlabel = \"Bbox Area % (As a perc of total image area)\", \n",
    "                ylabel = \"Number of bboxes\",\n",
    "                data_labels = [None],\n",
    "                data_list =[idict_test[\"bbox_area_percs\"]],\n",
    "                nbins=70,\n",
    "                use_logs=False,\n",
    "                edgecolor=None)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2759d9e3",
   "metadata": {},
   "source": [],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52b2bc6e",
   "metadata": {},
   "source": [],
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "98634befc514324259d6c9d42b6d889d9a27b32bc4075c57570577e494554633"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
